{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# baseline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 加载训练集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from pandas import DataFrame as DF\n",
    "from sklearn.feature_extraction.text import TfidfVectorizer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dir = \"../data/train.csv\"\n",
    "test_dir = \"../data/test.csv\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(train_dir, encoding=\"utf-8\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 数据清洗"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "def clean_text(text):\n",
    "    \"\"\"\n",
    "    Clean text\n",
    "    :param text: the string of text\n",
    "    :return: text string after cleaning\n",
    "    \"\"\"\n",
    "    # acronym\n",
    "    text = re.sub(r\"can\\'t\", \"can not\", text)\n",
    "    text = re.sub(r\"cannot\", \"can not \", text)\n",
    "    text = re.sub(r\"what\\'s\", \"what is\", text)\n",
    "    text = re.sub(r\"What\\'s\", \"what is\", text)\n",
    "    text = re.sub(r\"\\'ve \", \" have \", text)\n",
    "    text = re.sub(r\"n\\'t\", \" not \", text)\n",
    "    text = re.sub(r\"i\\'m\", \"i am \", text)\n",
    "    text = re.sub(r\"I\\'m\", \"i am \", text)\n",
    "    text = re.sub(r\"\\'re\", \" are \", text)\n",
    "    text = re.sub(r\"\\'d\", \" would \", text)\n",
    "    text = re.sub(r\"\\'ll\", \" will \", text)\n",
    "    text = re.sub(r\" e mail \", \" email \", text)\n",
    "    text = re.sub(r\" e \\- mail \", \" email \", text)\n",
    "    text = re.sub(r\" e\\-mail \", \" email \", text)\n",
    "\n",
    "    # spelling correction\n",
    "    text = re.sub(r\"ph\\.d\", \"phd\", text)\n",
    "    text = re.sub(r\"PhD\", \"phd\", text)\n",
    "    text = re.sub(r\" e g \", \" eg \", text)\n",
    "    text = re.sub(r\" fb \", \" facebook \", text)\n",
    "    text = re.sub(r\"facebooks\", \" facebook \", text)\n",
    "    text = re.sub(r\"facebooking\", \" facebook \", text)\n",
    "    text = re.sub(r\" usa \", \" america \", text)\n",
    "    text = re.sub(r\" us \", \" america \", text)\n",
    "    text = re.sub(r\" u s \", \" america \", text)\n",
    "    text = re.sub(r\" U\\.S\\. \", \" america \", text)\n",
    "    text = re.sub(r\" US \", \" america \", text)\n",
    "    text = re.sub(r\" American \", \" america \", text)\n",
    "    text = re.sub(r\" America \", \" america \", text)\n",
    "    text = re.sub(r\" mbp \", \" macbook-pro \", text)\n",
    "    text = re.sub(r\" mac \", \" macbook \", text)\n",
    "    text = re.sub(r\"macbook pro\", \"macbook-pro\", text)\n",
    "    text = re.sub(r\"macbook-pros\", \"macbook-pro\", text)\n",
    "    text = re.sub(r\" 1 \", \" one \", text)\n",
    "    text = re.sub(r\" 2 \", \" two \", text)\n",
    "    text = re.sub(r\" 3 \", \" three \", text)\n",
    "    text = re.sub(r\" 4 \", \" four \", text)\n",
    "    text = re.sub(r\" 5 \", \" five \", text)\n",
    "    text = re.sub(r\" 6 \", \" six \", text)\n",
    "    text = re.sub(r\" 7 \", \" seven \", text)\n",
    "    text = re.sub(r\" 8 \", \" eight \", text)\n",
    "    text = re.sub(r\" 9 \", \" nine \", text)\n",
    "    text = re.sub(r\"googling\", \" google \", text)\n",
    "    text = re.sub(r\"googled\", \" google \", text)\n",
    "    text = re.sub(r\"googleable\", \" google \", text)\n",
    "    text = re.sub(r\"googles\", \" google \", text)\n",
    "    text = re.sub(r\"dollars\", \" dollar \", text)\n",
    "\n",
    "    # punctuation\n",
    "    text = re.sub(r\"\\+\", \" + \", text)\n",
    "    text = re.sub(r\"'\", \" \", text)\n",
    "    text = re.sub(r\"-\", \" - \", text)\n",
    "    text = re.sub(r\"/\", \" / \", text)\n",
    "    text = re.sub(r\"\\\\\", \" \\ \", text)\n",
    "    text = re.sub(r\"=\", \" = \", text)\n",
    "    text = re.sub(r\"\\^\", \" ^ \", text)\n",
    "    text = re.sub(r\":\", \" : \", text)\n",
    "    text = re.sub(r\"\\.\", \" . \", text)\n",
    "    text = re.sub(r\",\", \" , \", text)\n",
    "    text = re.sub(r\"\\?\", \" ? \", text)\n",
    "    text = re.sub(r\"!\", \" ! \", text)\n",
    "    text = re.sub(r\"\\\"\", \" \\\" \", text)\n",
    "    text = re.sub(r\"&\", \" & \", text)\n",
    "    text = re.sub(r\"\\|\", \" | \", text)\n",
    "    text = re.sub(r\";\", \" ; \", text)\n",
    "    text = re.sub(r\"\\(\", \" ( \", text)\n",
    "    text = re.sub(r\"\\)\", \" ( \", text)\n",
    "\n",
    "    # symbol replacement\n",
    "    text = re.sub(r\"&\", \" and \", text)\n",
    "    text = re.sub(r\"\\|\", \" or \", text)\n",
    "    text = re.sub(r\"=\", \" equal \", text)\n",
    "    text = re.sub(r\"\\+\", \" plus \", text)\n",
    "    text = re.sub(r\"\\$\", \" dollar \", text)\n",
    "\n",
    "    # remove extra space\n",
    "    text = ' '.join(text.split())\n",
    "\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "train[\"text\"] = train.text.apply(clean_text)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### tf-idf 向量化"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "vectorizer = TfidfVectorizer(stop_words=\"english\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = vectorizer.fit_transform(train[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7613, 21364)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 朴素贝叶斯分类器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.naive_bayes import MultinomialNB\n",
    "from sklearn.model_selection import GridSearchCV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = MultinomialNB(fit_prior=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 交叉验证 网格搜索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 简单的网格搜索\n",
    "params = {\n",
    "    \"alpha\": [ 0.2517 + i * 0.00005 for i in range(50)]\n",
    "}\n",
    "\n",
    "grid = GridSearchCV(clf, params, scoring=\"f1\", n_jobs=4, cv=5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score=nan,\n",
       "             estimator=MultinomialNB(alpha=1.0, class_prior=None,\n",
       "                                     fit_prior=True),\n",
       "             iid='deprecated', n_jobs=4,\n",
       "             param_grid={'alpha': [0.2517, 0.25175, 0.25179999999999997,\n",
       "                                   0.25184999999999996, 0.25189999999999996,\n",
       "                                   0.25194999999999995, 0.252, 0.25205, 0.2521,\n",
       "                                   0.25215, 0.2522, 0.25225,\n",
       "                                   0.25229999999999997, 0.25234999999999996,\n",
       "                                   0.25239999999999996, 0.25244999999999995,\n",
       "                                   0.2525, 0.25255, 0.2526, 0.25265, 0.2527,\n",
       "                                   0.25275, 0.25279999999999997,\n",
       "                                   0.25284999999999996, 0.25289999999999996,\n",
       "                                   0.25294999999999995, 0.253, 0.25305, 0.2531,\n",
       "                                   0.25315, ...]},\n",
       "             pre_dispatch='2*n_jobs', refit=True, return_train_score=False,\n",
       "             scoring='f1', verbose=0)"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.fit(X_train, train.target)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.6548443351939582"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.best_score_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'alpha': 0.25184999999999996}"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.best_params_"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 目前最优的分类器"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = grid.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "MultinomialNB(alpha=0.25184999999999996, class_prior=None, fit_prior=True)"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "clf"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 加载测试集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {},
   "outputs": [],
   "source": [
    "test = pd.read_csv(test_dir)\n",
    "\n",
    "test[\"text\"] = test[\"text\"].apply(clean_text)\n",
    "\n",
    "X_test = vectorizer.transform(test[\"text\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3263, 21364)"
      ]
     },
     "execution_count": 58,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_test.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 在测试集上预测"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "y_test = clf.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3263,)"
      ]
     },
     "execution_count": 62,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([1, 1, 1, 1, 1], dtype=int64)"
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_test[0:5]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 写入结果并提交"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = DF({\"id\": test.id, \"target\": y_test})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [],
   "source": [
    "df.to_csv(\"./submit3.5.csv\", index=False, encoding=\"utf-8\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**baseline** 的 **F1 score** 为 **0.79652**"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
